﻿using SimpleCrawler.Entity;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using SimpleCrawler.Logger;
using log4net;
using SimpleCrawler.Tools;
using SimpleCrawler.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using SimpleCrawler.DAL;

namespace SimpleCrawler.Crawler
{
    //基础爬虫类
    public class CrawlerBase : ICrawler
    {
        /// <summary>
        /// 爬虫id 名称
        /// </summary>
        /// <param name="crawlerid"></param>
        /// <param name="sitename"></param>
        public CrawlerBase(int crawlerid, string sitename)
        {
            CrawlerID = crawlerid;
            SiteName = sitename;
            Filter = AppServiceProvider.ServiceProvider.GetService<BloomFilter<string>>();
        }

        /// <summary>
        /// 获取当前爬虫类别
        /// </summary>
        public Type TypeName { get { return this.GetType(); } }
        private ILog logger;
        /// <summary>
        /// 日志 记录html匹配错误
        /// </summary>
        public ILog Logger
        {
            get
            {
                if (logger == null)
                    logger = LoggerManager.GetLinkLogger(TypeName);
                return logger;
            }
        }
        /// <summary>
        /// 爬虫id
        /// </summary>
        public int CrawlerID { get; set; }
        /// <summary>
        /// 爬虫名称
        /// </summary>
        public string SiteName { get; set; }
        /// <summary>
        /// 每次分页请求的url
        /// </summary>
        public string RequestUrl { get; set; } = null;
        /// <summary>
        /// 当前任务
        /// </summary>
        public CrawlerTask CrawlerTask { get; set; }
        /// <summary>
        /// 是否监测url
        /// </summary>
        public bool CheckUrl { get; set; } = true;
        public BloomFilter<string> Filter { get; set; }
        public int  FilterCount { get; set; }

        /// <summary>
        /// 是否需要cookie
        /// </summary>
        public bool RequireCookie { get; set; } = false;
        /// <summary>
        /// cookie容器 保存每次请求的cookie
        /// </summary>
        public CookieContainer CookieContainer { get; set; }//定义Cookie容器

        #region 抓取逻辑 1请求url 2提取数据 3实现类中自定义处理数据 4是否有分页 有=>1 无=>抓取结束
        /// <summary>
        /// 设置抓取页入口
        /// </summary>
        /// <returns></returns>
        public virtual void GetStartedPageUrl()
        {

        }

        /// <summary>
        /// 分页逻辑 返回url地址
        /// </summary>
        /// <returns></returns>
        public virtual void GetNextPageUrl(int page)
        {
        }

        /// <summary>
        /// 获取目标页html
        /// </summary>
        /// <returns>html</returns>
        public virtual async Task<string> GetTargetHtmlString()
        {
            string pageSource = null;
            try
            {
                var request = (HttpWebRequest)WebRequest.Create(RequestUrl);
                request.Accept = "*/*";
                request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");//定义gzip压缩页面支持
                request.ContentType = "application/x-www-form-urlencoded";
                request.AllowAutoRedirect = false;
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
                request.Headers["Accept-Language"] = "zh-CN,zh;q=0.9";
                request.Referer = RequestUrl;
                //request.Headers["Host"] = RequestUrl;
                request.Timeout = 6000;
                request.KeepAlive = true;
                request.Method = "GET";
                if (RequireCookie && CookieContainer != null)
                {
                    request.CookieContainer = CookieContainer;
                }
                //if (proxy != null) request.Proxy = new WebProxy(proxy);//设置代理服务器IP，伪装请求地址
                //request.CookieContainer = this.CookiesContainer;//附加Cookie容器
                //request.ServicePoint.ConnectionLimit = int.MaxValue;//定义最大连接数

                using (var response = (HttpWebResponse)(await request.GetResponseAsync()))
                {
                    if (response.StatusCode != HttpStatusCode.OK)
                        return pageSource;

                    if (RequireCookie)
                    {
                        foreach (Cookie cookie in response.Cookies) this.CookieContainer.Add(cookie);
                    }

                    if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
                    {
                        using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {
                                pageSource = await reader.ReadToEndAsync();
                            }
                        }
                    }
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
                    {
                        using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {
                                pageSource = await reader.ReadToEndAsync();
                            }

                        }
                    }
                    else
                    {
                        using (Stream stream = response.GetResponseStream())//原始
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {

                                pageSource = await reader.ReadToEndAsync();
                            }
                        }
                    }
                }
                request.Abort();
            }
            catch (Exception ex)
            {
                throw ex;
            }
            return pageSource;
        }

        /// <summary>
        /// 处理html,提取数据
        /// </summary>
        /// <param name="html">html</param>
        /// <returns>保存数据条数,负数表示跳过此页,整数为抓取条数 0则匹配错误或IP被加入黑名单</returns>
        public async virtual Task<int> GetResultContent(string html)
        {
            return 0;
        }
        #endregion

        #region 封装请求,过滤等辅助方法
        //https请求
        public static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
        {
            return true;
        }
        /// <summary>
        /// get请求 返回html
        /// </summary>
        /// <param name="url"></param>
        /// <param name="headers"></param>
        /// <param name="saveCookie"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public async Task<string> SendRequestByGet(string url, Hashtable headers = null, bool saveCookie = false, string encoding = "utf-8")
        {

            string page = string.Empty;
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Accept = "*/*";
            request.ContentType = "application/x-www-form-urlencoded";
            request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
            request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
            request.Headers["Accept-Language"] = "zh-CN,zh;q=0.9";
            //request.Headers["Referer"] = url;
            request.Method = "GET";
            if (RequireCookie && CookieContainer != null)
            {
                request.CookieContainer = CookieContainer;
            }

            if (headers != null)
            {
                foreach (DictionaryEntry kw in headers)
                {
                    request.Headers[kw.Key.ToString()] = kw.Value.ToString();
                }
            }

            Encoding encoder = Encoding.GetEncoding(encoding);
            using (var response = (HttpWebResponse)(await request.GetResponseAsync()))
            {
                if (saveCookie)
                {
                    foreach (Cookie cookie in response.Cookies)
                        this.CookieContainer.Add(cookie);
                }

                if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
                {
                    using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }
                    }
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
                {
                    using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }

                    }
                }
                else
                {
                    using (Stream stream = response.GetResponseStream())
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }
                    }
                }
            }
            request.Abort();
            return page;
        }
        /// <summary>
        /// get请求 无需返回html
        /// </summary>
        /// <param name="url"></param>
        /// <param name="headers"></param>
        /// <param name="saveCookie"></param>
        /// <returns></returns>
        public async Task SendRequestByGetWithoutHtml(string url, Hashtable headers = null, bool saveCookie = false)
        {

            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Accept = "*/*";
            request.ContentType = "application/x-www-form-urlencoded";
            request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
            request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
            request.Headers["Accept-Language"] = "zh-CN,zh;q=0.9";
            request.Headers["Referer"] = url;
            request.Method = "GET";
            if (RequireCookie && CookieContainer != null)
            {
                request.CookieContainer = CookieContainer;
            }

            if (headers != null)
            {
                foreach (DictionaryEntry kw in headers)
                {
                    request.Headers[kw.Key.ToString()] = kw.Value.ToString();
                }
            }

            using (var response = (HttpWebResponse)(await request.GetResponseAsync()))
            {
                if (saveCookie)
                {
                    foreach (Cookie cookie in response.Cookies)
                        this.CookieContainer.Add(cookie);
                }
            }
            request.Abort();
        }
        /// <summary>
        /// post请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="content"></param>
        /// <param name="headers"></param>
        /// <param name="saveCookie"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public async Task<string> SendRequestByPost(string url, string content, Hashtable headers = null, bool saveCookie = false, string encoding = "utf-8")
        {

            string page = string.Empty;
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Accept = "*/*";
            request.ContentType = "application/x-www-form-urlencoded";
            request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
            request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
            request.Headers["Accept-Language"] = "zh-CN,zh;q=0.9";
            //request.Headers["Origin"] = "https://www.tianyancha.com";
            //request.Headers["X-Requested-With"] = "XMLHttpRequest";
            //request.Referer = "https://www.tianyancha.com/";
            request.Headers["Referer"] = url;
            request.Method = "POST";
            if (RequireCookie && CookieContainer != null)
            {
                request.CookieContainer = CookieContainer;
            }
            if (headers != null)
            {
                foreach (DictionaryEntry kw in headers)
                {
                    request.Headers[kw.Key.ToString()] = kw.Value.ToString();
                }
            }

            byte[] postdatabytes = Encoding.GetEncoding("utf-8").GetBytes(content);
            request.ContentLength = postdatabytes.Length;
            using (Stream stream = await request.GetRequestStreamAsync())
            {
                await stream.WriteAsync(postdatabytes, 0, postdatabytes.Length);
                stream.Flush();
                stream.Close();
            }

            Encoding encoder = Encoding.GetEncoding(encoding);
            using (var response = (HttpWebResponse)(await request.GetResponseAsync()))
            {
                if (saveCookie)
                {
                    foreach (Cookie cookie in response.Cookies)
                        this.CookieContainer.Add(cookie);
                }

                if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
                {
                    using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }
                    }
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
                {
                    using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }

                    }
                }
                else
                {
                    using (Stream stream = response.GetResponseStream())
                    {
                        using (StreamReader reader = new StreamReader(stream, encoder))
                        {
                            page = await reader.ReadToEndAsync();
                        }
                    }
                }
            }
            request.Abort();
            return page;
        }
        /// <summary>
        /// 去除所有<>标记
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public string CleanTag(string html)
        {
            string pattern = @"<[\s\S]*?>";
            html = Regex.Replace(html, pattern, string.Empty, RegexOptions.IgnoreCase);
            return html;
        }
        #endregion


    }
}
